#/usr/bin/env python
import sys
import json
from urllib.request import Request
from urllib.request import urlopen
from bs4 import BeautifulSoup as BS

from bspage import BSPage as BP


ipshu_region = 'https://en.ipshu.com/country_region/%s/%s'
ipshu_city = 'https://en.ipshu.com/country_region_city/AD/Canillo/Canillo'
header = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36'
}

try:
    _, country, state = sys.argv[:3]
except:
    print('%s country state' % sys.argv[0])
    sys.exit()

url = 'https://en.ipshu.com/country_region/%s/%s?page=' % (country, state.replace(' ', '%20'))
iplist = []
ipheader = ['From', 'To', 'City']
page = 0
while 1:
    u = url + str(page)
    print(u)
    req = Request(u, headers=header)
    rep = urlopen(req).read()
    # print(rep)
    ips = BP.parse_table(rep, ipheader)
    iplist += ips
    print(ips)
    bs = BS(rep, 'html5lib')
    next = bs.find('li', class_='pager-next last')
    if not next or not next.find('a'):
        break
    page += 1
print(len(iplist))
with open(country + '_' + state + '.json', 'w') as fp:
    fp.write(json.dumps(iplist, ensure_ascii=False))

# print(BP.parse_table(rep, ['From', 'To', 'City']))
